﻿using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;

namespace g1_c
{
    public class Crawler
    {
        public string startUrl;
        private Hashtable urls = new Hashtable();
        private int count = 0;

        public Crawler(string startUrl)
        {
            this.urls.Add(startUrl, false);
            this.startUrl = startUrl;
        }

        public void Crawl()
        {
            Console.WriteLine("开始爬行了......");
            string html = Download(startUrl);
            Console.WriteLine("爬行" + startUrl + "页面！");
            urls[startUrl] = true;
            Parse(html);//对首页内容进行分析

            while (true)
            {
                string current = null;
                foreach (string url in urls.Keys)
                {
                    if ((bool)urls[url]) continue;
                    current = url;
                }
                if (current == null || count > 10) break;
                Console.WriteLine("爬行" + current + "页面！");
                html = Download(current);
                urls[current] = true;
                count++;
                Console.WriteLine(count);
            }
        }
        public string Download(string url)
        {//下载指定网页的内容
            try
            {
                WebClient webClient = new();
                webClient.Encoding = Encoding.UTF8; //设置编码格式
                string html = webClient.DownloadString(url);
                string fileName = count.ToString() + ".html";
                File.WriteAllText(fileName, html, Encoding.UTF8);
                return html;
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.Message);
                return "";
            }
        }
        public void Parse(string html)
        {
            string strRef = @"(href|HREF)[]*=[]*[""'][^""'#>]+.html[""']"; //只匹配Html
            MatchCollection matches = new Regex(strRef).Matches(html); //所有匹配成功的String
            foreach (Match match in matches)
            {
                //Console.WriteLine(match.Value);
                strRef = match.Value.Substring(match.Value.IndexOf('=') + 1).Trim('"', '\"', '#', ' ', '>');
                Console.WriteLine("before: " + strRef);
                strRef = Transform(strRef);
                Console.WriteLine("after: " + strRef);
                if (strRef.Length == 0) continue;
                if (urls[strRef] == null) urls[strRef] = false;
            }
        }

        public string Transform(string url)
        {//相对路径转换成绝对路径
            string[] arr = url.Split("//");
            if (arr[0] == "https:"|| arr[0] == "http:")
                return url;
            else if (arr[0] == "")
                return "https:" + url;
            else
                return UrlCut("/") + url;

        }
        public string UrlCut(string cut)
        {
            string[] arr = startUrl.Split("//");
            string[] arr2 = arr[arr.Length - 1].Split('/');
            string res = "";
            if (arr[0] == "https:")
                res = arr[0] + "/";
            int n = 0;
            if (cut == "/") n = 1;
            if (cut == "./") n = 2;
            for (int i = 0; i < arr2.Length - n; i++)
            {
                res += "/" + arr2[i];
            }
            return res;
        }
    }
}
